#!/usr/bin/python
#百度爬虫的更新版本
from bs4 import BeautifulSoup
from urllib import request,parse
import threading
from queue import Queue
import time

class spider(threading.Thread):
    def __init__(self,q):
        threading.Thread.__init__(self)
        self.q=q
    def run(self):
        while not self.q.empty():
            url=self.q.get()
            try:
                self.spidering(url)
            except:
                print(Exception)
    def spidering(self,url):
        req=request.Request(url)
        req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:54.0) Gecko/20100101 Firefox/54.0')
        try:
            openPage=request.urlopen(req,timeout=5)
        except:
            print("连接失败，百度url失效或拒绝访问！","---->",self.page)
        # ...得到一个beautifulsoup对象
        bes=BeautifulSoup(openPage,"html.parser")
        h3 = bes.findAll("h3")
        for h in h3:
            try:
                _url=request.urlopen(h.contents[0]["href"],timeout=5).geturl()
                print(_url)
                if _url not in open('urllist.txt').read():
                    self.save_txt(_url)
            except:
                print("获取url失败，可能原因是网址失效！")
    def save_txt(self,string):
        try:
            txt = open("urllist.txt", "a")
            txt.write(string+"\n")
            txt.close()
        except:
            print("文件操作失误！")


def main():
    q=Queue()
    searchName=input("请输入搜索内容：")
    for i in range(76):
        url='https://www.baidu.com/s?wd='+searchName+'&pn='+str(i)+'0'
        q.put(url)

    threads=[]
    thread_num=76
    for i in range(thread_num):
        threads.append(spider(q))
    for t in threads:
        t.start()
        #time.sleep(1)   #要防止遇到验证码，可以把前面注释去掉，会导致速度变慢
    for t in threads:
        t.join()



if __name__ == '__main__':

    main()
